{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {},
  "cells": [
    {
      "metadata": {},
      "source": [
        "<td>\n",
        "   <a target=\"_blank\" href=\"https://labelbox.com\" ><img src=\"https://labelbox.com/blog/content/images/2021/02/logo-v4.svg\" width=256/></a>\n",
        "</td>\n"
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "<td>\n",
        "<a href=\"https://colab.research.google.com/github/Labelbox/labelbox-python/blob/master/examples/integrations/sam/meta_sam_labelbox_video.ipynb\" target=\"_blank\"><img\n",
        "src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"></a>\n",
        "</td>\n",
        "\n",
        "<td>\n",
        "<a href=\"https://github.com/Labelbox/labelbox-python/blob/master/examples/integrations/sam/meta_sam_labelbox_video.ipynb\" target=\"_blank\"><img\n",
        "src=\"https://img.shields.io/badge/GitHub-100000?logo=github&logoColor=white\" alt=\"GitHub\"></a>\n",
        "</td>"
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "# Setting the stage\n",
        "\n",
        "First, we import and prepare the prerequisites to process the video."
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "### General dependencies"
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "!nvidia-smi"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "import os\n",
        "HOME = os.getcwd()\n",
        "print(HOME)\n",
        "\n",
        "import sys\n",
        "from google.colab.patches import cv2_imshow\n",
        "import cv2\n",
        "import PIL\n",
        "from PIL import Image\n",
        "import numpy as np\n",
        "import uuid\n",
        "import tempfile\n",
        "\n",
        "from IPython import display\n",
        "display.clear_output()\n",
        "from IPython.display import display, Image\n",
        "from io import BytesIO"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# You can also use the Labelbox Client API to get specific videos or an entire\n",
        "# dataset from your Catalog. Refer to these docs:\n",
        "# https://labelbox-python.readthedocs.io/en/latest/#labelbox.client.Client.get_data_row\n",
        "\n",
        "VIDEO_PATH = \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/skateboarding.mp4\"\n",
        "\n",
        "%cd {HOME}\n",
        "!wget -v {VIDEO_PATH}"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "### YOLOv8 dependencies"
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "# Dependencies for YOLOv8\n",
        "\n",
        "!pip install ultralytics==8.0.20"
      ],
      "cell_type": "code",
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
            "Collecting ultralytics==8.0.20\n",
            "  Downloading ultralytics-8.0.20-py3-none-any.whl.metadata (24 kB)\n",
            "Requirement already satisfied: matplotlib>=3.2.2 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (3.3.3)\n",
            "Requirement already satisfied: numpy>=1.18.5 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (1.26.2)\n",
            "Requirement already satisfied: opencv-python>=4.6.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (4.8.1.78)\n",
            "Requirement already satisfied: Pillow>=7.1.2 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (10.1.0)\n",
            "Requirement already satisfied: PyYAML>=5.3.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (6.0.1)\n",
            "Requirement already satisfied: requests>=2.23.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (2.31.0)\n",
            "Requirement already satisfied: scipy>=1.4.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (1.12.0)\n",
            "Requirement already satisfied: torch>=1.7.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (2.2.0)\n",
            "Requirement already satisfied: torchvision>=0.8.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (0.17.0)\n",
            "Requirement already satisfied: tqdm>=4.64.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (4.66.1)\n",
            "Requirement already satisfied: tensorboard>=2.4.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (2.15.2)\n",
            "Collecting pandas>=1.1.4 (from ultralytics==8.0.20)\n",
            "  Downloading pandas-2.2.1-cp310-cp310-macosx_11_0_arm64.whl.metadata (19 kB)\n",
            "Collecting seaborn>=0.11.0 (from ultralytics==8.0.20)\n",
            "  Downloading seaborn-0.13.2-py3-none-any.whl.metadata (5.4 kB)\n",
            "Requirement already satisfied: ipython in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (8.18.1)\n",
            "Requirement already satisfied: psutil in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ultralytics==8.0.20) (5.9.6)\n",
            "Collecting thop>=0.1.1 (from ultralytics==8.0.20)\n",
            "  Downloading thop-0.1.1.post2209072238-py3-none-any.whl.metadata (2.7 kB)\n",
            "Collecting sentry-sdk (from ultralytics==8.0.20)\n",
            "  Downloading sentry_sdk-1.42.0-py2.py3-none-any.whl.metadata (9.8 kB)\n",
            "Requirement already satisfied: cycler>=0.10 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from matplotlib>=3.2.2->ultralytics==8.0.20) (0.12.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from matplotlib>=3.2.2->ultralytics==8.0.20) (1.4.5)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from matplotlib>=3.2.2->ultralytics==8.0.20) (3.1.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from matplotlib>=3.2.2->ultralytics==8.0.20) (2.8.2)\n",
            "Collecting pytz>=2020.1 (from pandas>=1.1.4->ultralytics==8.0.20)\n",
            "  Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n",
            "Collecting tzdata>=2022.7 (from pandas>=1.1.4->ultralytics==8.0.20)\n",
            "  Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from requests>=2.23.0->ultralytics==8.0.20) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from requests>=2.23.0->ultralytics==8.0.20) (2.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from requests>=2.23.0->ultralytics==8.0.20) (1.25.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from requests>=2.23.0->ultralytics==8.0.20) (2020.6.20)\n",
            "Collecting matplotlib>=3.2.2 (from ultralytics==8.0.20)\n",
            "  Downloading matplotlib-3.8.3-cp310-cp310-macosx_11_0_arm64.whl.metadata (5.8 kB)\n",
            "Collecting contourpy>=1.0.1 (from matplotlib>=3.2.2->ultralytics==8.0.20)\n",
            "  Downloading contourpy-1.2.0-cp310-cp310-macosx_11_0_arm64.whl.metadata (5.8 kB)\n",
            "Collecting fonttools>=4.22.0 (from matplotlib>=3.2.2->ultralytics==8.0.20)\n",
            "  Downloading fonttools-4.49.0-cp310-cp310-macosx_10_9_universal2.whl.metadata (159 kB)\n",
            "\u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m159.1/159.1 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from matplotlib>=3.2.2->ultralytics==8.0.20) (23.2)\n",
            "Requirement already satisfied: absl-py>=0.4 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (2.1.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (1.62.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (2.24.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (1.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (3.5.2)\n",
            "Requirement already satisfied: protobuf!=4.24.0,>=3.19.6 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (4.25.1)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (68.2.2)\n",
            "Requirement already satisfied: six>1.9 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (1.15.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from tensorboard>=2.4.1->ultralytics==8.0.20) (3.0.1)\n",
            "Requirement already satisfied: filelock in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from torch>=1.7.0->ultralytics==8.0.20) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from torch>=1.7.0->ultralytics==8.0.20) (4.9.0)\n",
            "Requirement already satisfied: sympy in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from torch>=1.7.0->ultralytics==8.0.20) (1.12)\n",
            "Requirement already satisfied: networkx in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from torch>=1.7.0->ultralytics==8.0.20) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from torch>=1.7.0->ultralytics==8.0.20) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from torch>=1.7.0->ultralytics==8.0.20) (2023.12.2)\n",
            "Requirement already satisfied: decorator in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ipython->ultralytics==8.0.20) (5.1.1)\n",
            "Requirement already satisfied: jedi>=0.16 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ipython->ultralytics==8.0.20) (0.19.1)\n",
            "Requirement already satisfied: matplotlib-inline in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ipython->ultralytics==8.0.20) (0.1.6)\n",
            "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ipython->ultralytics==8.0.20) (3.0.41)\n",
            "Requirement already satisfied: pygments>=2.4.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ipython->ultralytics==8.0.20) (2.17.2)\n",
            "Requirement already satisfied: stack-data in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ipython->ultralytics==8.0.20) (0.6.3)\n",
            "Requirement already satisfied: traitlets>=5 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ipython->ultralytics==8.0.20) (5.14.0)\n",
            "Requirement already satisfied: exceptiongroup in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ipython->ultralytics==8.0.20) (1.2.0)\n",
            "Requirement already satisfied: pexpect>4.3 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from ipython->ultralytics==8.0.20) (4.9.0)\n",
            "Collecting urllib3<3,>=1.21.1 (from requests>=2.23.0->ultralytics==8.0.20)\n",
            "  Downloading urllib3-2.2.1-py3-none-any.whl.metadata (6.4 kB)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.4.1->ultralytics==8.0.20) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.4.1->ultralytics==8.0.20) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.4.1->ultralytics==8.0.20) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from google-auth-oauthlib<2,>=0.5->tensorboard>=2.4.1->ultralytics==8.0.20) (1.3.1)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from jedi>=0.16->ipython->ultralytics==8.0.20) (0.8.3)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from pexpect>4.3->ipython->ultralytics==8.0.20) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython->ultralytics==8.0.20) (0.2.12)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from werkzeug>=1.0.1->tensorboard>=2.4.1->ultralytics==8.0.20) (2.1.3)\n",
            "Requirement already satisfied: executing>=1.2.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from stack-data->ipython->ultralytics==8.0.20) (2.0.1)\n",
            "Requirement already satisfied: asttokens>=2.1.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from stack-data->ipython->ultralytics==8.0.20) (2.4.1)\n",
            "Requirement already satisfied: pure-eval in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from stack-data->ipython->ultralytics==8.0.20) (0.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from sympy->torch>=1.7.0->ultralytics==8.0.20) (1.3.0)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.4.1->ultralytics==8.0.20) (0.5.1)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /Users/gabrielunderwood/repos/labelbox-python/env/lib/python3.10/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard>=2.4.1->ultralytics==8.0.20) (3.2.2)\n",
            "Downloading ultralytics-8.0.20-py3-none-any.whl (261 kB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m261.2/261.2 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading pandas-2.2.1-cp310-cp310-macosx_11_0_arm64.whl (11.3 MB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m11.3/11.3 MB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
            "\u001b[?25hDownloading seaborn-0.13.2-py3-none-any.whl (294 kB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m294.9/294.9 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading matplotlib-3.8.3-cp310-cp310-macosx_11_0_arm64.whl (7.5 MB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m7.5/7.5 MB\u001b[0m \u001b[31m27.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading thop-0.1.1.post2209072238-py3-none-any.whl (15 kB)\n",
            "Downloading sentry_sdk-1.42.0-py2.py3-none-any.whl (263 kB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m263.5/263.5 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading contourpy-1.2.0-cp310-cp310-macosx_11_0_arm64.whl (242 kB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m242.2/242.2 kB\u001b[0m \u001b[31m33.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fonttools-4.49.0-cp310-cp310-macosx_10_9_universal2.whl (2.8 MB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m2.8/2.8 MB\u001b[0m \u001b[31m19.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m505.5/505.5 kB\u001b[0m \u001b[31m23.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m345.4/345.4 kB\u001b[0m \u001b[31m34.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading urllib3-2.2.1-py3-none-any.whl (121 kB)\n",
            "\u001b[2K   \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m121.1/121.1 kB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pytz, urllib3, tzdata, fonttools, contourpy, sentry-sdk, pandas, matplotlib, thop, seaborn, ultralytics\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.25.10\n",
            "    Uninstalling urllib3-1.25.10:\n",
            "      Successfully uninstalled urllib3-1.25.10\n",
            "  Attempting uninstall: matplotlib\n",
            "    Found existing installation: matplotlib 3.3.3\n",
            "    Uninstalling matplotlib-3.3.3:\n",
            "      Successfully uninstalled matplotlib-3.3.3\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "nvidia-tlt 0.1.21 requires requests==2.24.0, but you have requests 2.31.0 which is incompatible.\n",
            "nvidia-tlt 0.1.21 requires urllib3==1.25.10, but you have urllib3 2.2.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed contourpy-1.2.0 fonttools-4.49.0 matplotlib-3.8.3 pandas-2.2.1 pytz-2024.1 seaborn-0.13.2 sentry-sdk-1.42.0 thop-0.1.1.post2209072238 tzdata-2024.1 ultralytics-8.0.20 urllib3-2.2.1\n"
          ]
        }
      ],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Import packages\n",
        "\n",
        "import ultralytics\n",
        "ultralytics.checks()\n",
        "from ultralytics import YOLO"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Instantiate YOLOv8 model\n",
        "\n",
        "model = YOLO(f'{HOME}/yolov8n.pt')\n",
        "colors = np.random.randint(0, 256, size=(len(model.names), 3))\n",
        "\n",
        "print(model.names)\n",
        "\n",
        "# Specify which classes you care about. The rest of classes will be filtered out.\n",
        "chosen_class_ids = [0] # person"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "### SAM dependencies"
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "# Download SAM model SDK\n",
        "\n",
        "%cd {HOME}\n",
        "!{sys.executable} -m pip install 'git+https://github.com/facebookresearch/segment-anything.git'"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Download SAM model weights\n",
        "\n",
        "%cd {HOME}\n",
        "!mkdir {HOME}/weights\n",
        "%cd {HOME}/weights\n",
        "\n",
        "!wget -q https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth\n",
        "\n",
        "CHECKPOINT_PATH = os.path.join(HOME, \"weights\", \"sam_vit_h_4b8939.pth\")\n",
        "print(CHECKPOINT_PATH, \"; exist:\", os.path.isfile(CHECKPOINT_PATH))"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Import packages\n",
        "\n",
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "from segment_anything import sam_model_registry, SamAutomaticMaskGenerator, SamPredictor"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Instantiate SAM model\n",
        "\n",
        "DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "sam = sam_model_registry[\"vit_h\"](checkpoint=CHECKPOINT_PATH).to(device=DEVICE)\n",
        "mask_predictor = SamPredictor(sam)"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "### Labelbox dependencies"
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "# Install labelbox package\n",
        "\n",
        "!pip install -q \"labelbox[data]\""
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Import packages\n",
        "\n",
        "import labelbox as lb\n",
        "import labelbox.types as lb_types"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Create a Labelbox API key for your account by following the instructions here:\n",
        "# https://docs.labelbox.com/reference/create-api-key\n",
        "# Then, fill it in here\n",
        "\n",
        "API_KEY = \"\"\n",
        "client = lb.Client(API_KEY)"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "### Helper functions"
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "# Cast color to ints\n",
        "def get_color(color):\n",
        "  return (int(color[0]), int(color[1]), int(color[2]))\n",
        "\n",
        "# Get video dimensions\n",
        "def get_video_dimensions(input_cap):\n",
        "  width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
        "  height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
        "  return height, width\n",
        "\n",
        "# Get output video writer with same dimensions and fps as input video\n",
        "def get_output_video_writer(input_cap, output_path):\n",
        "  # Get the video's properties (width, height, FPS)\n",
        "  width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
        "  height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
        "  fps = int(cap.get(cv2.CAP_PROP_FPS))\n",
        "\n",
        "  # Define the output video file\n",
        "  output_codec = cv2.VideoWriter_fourcc(*\"mp4v\")  # MP4 codec\n",
        "  output_video = cv2.VideoWriter(output_path, output_codec, fps, (width, height))\n",
        "\n",
        "  return output_video\n",
        "\n",
        "# Visualize a video frame with bounding boxes, classes and confidence scores\n",
        "def visualize_detections(frame, boxes, conf_thresholds, class_ids):\n",
        "    frame_copy = np.copy(frame)\n",
        "    for idx in range(len(boxes)):\n",
        "        class_id = int(class_ids[idx])\n",
        "        conf = float(conf_thresholds[idx])\n",
        "        x1, y1, x2, y2 = int(boxes[idx][0]), int(boxes[idx][1]), int(boxes[idx][2]), int(boxes[idx][3])\n",
        "        color = colors[class_id]\n",
        "        label = f\"{model.names[class_id]}: {conf:.2f}\"\n",
        "        cv2.rectangle(frame_copy, (x1, y1), (x2, y2), get_color(color), 2)\n",
        "        cv2.putText(frame_copy, label, (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, get_color(color), 2)\n",
        "    return frame_copy\n",
        "\n",
        "def add_color_to_mask(mask, color):\n",
        "  next_mask = mask.astype(np.uint8)\n",
        "  next_mask = np.expand_dims(next_mask, 0).repeat(3, axis=0)\n",
        "  next_mask = np.moveaxis(next_mask, 0, -1)\n",
        "  return next_mask * color\n",
        "\n",
        "# Merge masks into a single, multi-colored mask\n",
        "def merge_masks_colored(masks, class_ids):\n",
        "  filtered_class_ids = []\n",
        "  filtered_masks = []\n",
        "  for idx, cid in enumerate(class_ids):\n",
        "    if int(cid) in chosen_class_ids:\n",
        "      filtered_class_ids.append(cid)\n",
        "      filtered_masks.append(masks[idx])\n",
        "\n",
        "  merged_with_colors = add_color_to_mask(filtered_masks[0][0], get_color(colors[int(filtered_class_ids[0])])).astype(np.uint8)\n",
        "\n",
        "  if len(filtered_masks) == 1:\n",
        "    return merged_with_colors\n",
        "\n",
        "  for i in range(1, len(filtered_masks)):\n",
        "    curr_mask_with_colors = add_color_to_mask(filtered_masks[i][0], get_color(colors[int(filtered_class_ids[i])]))\n",
        "    merged_with_colors = np.bitwise_or(merged_with_colors, curr_mask_with_colors)\n",
        "\n",
        "  return merged_with_colors.astype(np.uint8)\n",
        "\n",
        "def get_instance_uri(client, global_key, array):\n",
        "    \"\"\" Reads a numpy array into a temp Labelbox data row to-be-uploaded to Labelbox\n",
        "    Args:\n",
        "        client        :   Required (lb.Client) - Labelbox Client object\n",
        "        global_key    :   Required (str) - Data row global key\n",
        "        array         :   Required (np.ndarray) - NumPy ndarray representation of an image\n",
        "    Returns:\n",
        "        Temp Labelbox data row to-be-uploaded to Labelbox as row data\n",
        "    \"\"\"\n",
        "    # Convert array to PIL image\n",
        "    image_as_pil = PIL.Image.fromarray(array)\n",
        "    # Convert PIL image to PNG file bytes\n",
        "    image_as_bytes = BytesIO()\n",
        "    image_as_pil.save(image_as_bytes, format='PNG')\n",
        "    image_as_bytes = image_as_bytes.getvalue()\n",
        "    # Convert PNG file bytes to a temporary Labelbox URL\n",
        "    url = client.upload_data(\n",
        "        content=image_as_bytes, \n",
        "        filename=global_key,\n",
        "        content_type=\"image/jpeg\",\n",
        "        sign=True\n",
        "    )\n",
        "    # Return the URL\n",
        "    return url\n",
        "\n",
        "def get_local_instance_uri(array):\n",
        "    # Convert array to PIL image\n",
        "    image_as_pil = PIL.Image.fromarray(array)\n",
        "\n",
        "    with tempfile.NamedTemporaryFile(suffix='.png', dir=\"/content\", delete=False) as temp_file:\n",
        "      image_as_pil.save(temp_file)\n",
        "      file_name = temp_file.name\n",
        "\n",
        "    # Return the URL\n",
        "    return file_name\n",
        "\n",
        "def create_mask_frame(frame_num, instance_uri):\n",
        "  return lb_types.MaskFrame(index=frame_num, instance_uri=instance_uri)\n",
        "\n",
        "def create_mask_instances(class_ids):\n",
        "  instances = []\n",
        "  for cid in list(set(class_ids)): # get unique class ids\n",
        "    if int(cid) in chosen_class_ids:\n",
        "      color = get_color(colors[int(cid)])\n",
        "      name = model.names[int(cid)]\n",
        "      instances.append(lb_types.MaskInstance(color_rgb=color, name=name))\n",
        "  return instances\n",
        "\n",
        "def create_video_mask_annotation(frames, instance):\n",
        "  return lb_types.VideoMaskAnnotation(\n",
        "        frames=frames,\n",
        "        instances=[instance]\n",
        "    )"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "### Labelbox setup"
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "# Create a new dataset\n",
        "\n",
        "# read more here: https://docs.labelbox.com/reference/data-row-global-keys\n",
        "global_key = os.path.basename(VIDEO_PATH)\n",
        "\n",
        "asset = {\n",
        "    \"row_data\": VIDEO_PATH,\n",
        "    \"global_key\": global_key,\n",
        "    #\"media_type\": \"VIDEO\"\n",
        "}\n",
        "\n",
        "dataset = client.create_dataset(name=\"yolo-sam-video-masks-dataset\")\n",
        "task = dataset.create_data_rows([asset])\n",
        "task.wait_till_done()\n",
        "\n",
        "print(f\"Errors: {task.errors}\")\n",
        "print(f\"Failed data rows: {task.failed_data_rows}\")"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Run through YOLOv8 on the video once quickly to get unique class ids present\n",
        "# This will inform which classes we add to the ontology\n",
        "\n",
        "cap = cv2.VideoCapture(VIDEO_PATH)\n",
        "\n",
        "unique_class_ids = set()\n",
        "\n",
        "# Loop through the frames of the video\n",
        "frame_num = 1\n",
        "while cap.isOpened():\n",
        "  if frame_num % 30 == 0 or frame_num == 1:\n",
        "    print(\"Processing frame number\", frame_num)\n",
        "  ret, frame = cap.read()\n",
        "  if not ret:\n",
        "      break\n",
        "\n",
        "  # Run frame through YOLOv8 and get class ids predicted\n",
        "  detections = model.predict(frame, conf=0.7) # frame is a numpy array\n",
        "  for cid in detections[0].boxes.cls:\n",
        "    unique_class_ids.add(int(cid))\n",
        "  frame_num += 1\n",
        "\n",
        "cap.release()"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "unique_class_ids"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Create a new ontology if you don't have one\n",
        "\n",
        "# Add all chosen classes into the ontology\n",
        "tools = []\n",
        "for cls in chosen_class_ids:\n",
        "  tools.append(lb.Tool(tool=lb.Tool.Type.RASTER_SEGMENTATION, name=model.names[cls]))\n",
        "\n",
        "ontology_builder = lb.OntologyBuilder(\n",
        "    classifications=[],\n",
        "    tools=tools\n",
        "  )\n",
        "\n",
        "ontology = client.create_ontology(\"yolo-sam-video-masks-ontology\",\n",
        "                                  ontology_builder.asdict()\n",
        "                                  #media_type=lb.MediaType.Image\n",
        "                                  )\n",
        "\n",
        "# Or get an existing ontology by name or ID (uncomment one of the below)\n",
        "\n",
        "# ontology = client.get_ontologies(\"Demo Chair\").get_one()\n",
        "\n",
        "# ontology = client.get_ontology(\"clhee8kzt049v094h7stq7v25\")"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Create a new project if you don't have one\n",
        "\n",
        "# Project defaults to batch mode with benchmark quality settings if this argument is not provided\n",
        "# Queue mode will be deprecated once dataset mode is deprecated\n",
        "project = client.create_project(name=\"yolo-sam-video-masks-project\",\n",
        "                                media_type=lb.MediaType.Video)\n",
        "\n",
        "# Or get an existing project by ID (uncomment the below)\n",
        "\n",
        "# project = get_project(\"fill_in_project_id\")\n",
        "\n",
        "# If the project already has an ontology set up, comment out this line\n",
        "project.setup_editor(ontology)"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Create a new batch of data for the project you specified above\n",
        "\n",
        "# Uncomment if you are using `data_rows` parameter below\n",
        "# data_row_ids = client.get_data_row_ids_for_global_keys([global_key])['results']\n",
        "\n",
        "batch = project.create_batch(\n",
        "    \"yolo-sam-video-masks-project\",  # each batch in a project must have a unique name\n",
        "    \n",
        "    # you can also specify global_keys instead of data_rows\n",
        "    global_keys=[global_key],\n",
        "\n",
        "    # you can also specify data_rows instead of global_keys\n",
        "    #data_rows=data_row_ids,\n",
        "    \n",
        "    priority=1  # priority between 1(highest) - 5(lowest)\n",
        ")\n",
        "\n",
        "print(f\"Batch: {batch}\")"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "tools = ontology.tools()\n",
        "\n",
        "feature_schema_ids = dict()\n",
        "for tool in tools:\n",
        "  feature_schema_ids[tool.name] = tool.feature_schema_id\n",
        "\n",
        "print(feature_schema_ids)"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "### Loop through each frame of video and process it"
      ],
      "cell_type": "markdown"
    },
    {
      "metadata": {},
      "source": [
        "# Run YOLOv8 and then SAM on each frame, and write visualization videos to disk\n",
        "# You can download /content/skateboarding_boxes.mp4 and /content/skateboarding_masks.mp4\n",
        "# to visualize the results\n",
        "\n",
        "cap = cv2.VideoCapture(VIDEO_PATH)\n",
        "\n",
        "output_video_boxes = get_output_video_writer(cap, \"/content/skateboarding_boxes.mp4\")\n",
        "output_video_masks = get_output_video_writer(cap, \"/content/skateboarding_masks.mp4\")\n",
        "mask_frames = []\n",
        "\n",
        "# Loop through the frames of the video\n",
        "frame_num = 1\n",
        "while cap.isOpened():\n",
        "  if frame_num % 30 == 0 or frame_num == 1:\n",
        "    print(\"Processing frames\", frame_num, \"-\", frame_num+29)\n",
        "  ret, frame = cap.read()\n",
        "  if not ret:\n",
        "      break\n",
        "\n",
        "  # Run frame through YOLOv8 to get detections\n",
        "  detections = model.predict(frame, conf=0.7) # frame is a numpy array\n",
        "  \n",
        "  # Write detections to output video\n",
        "  frame_with_detections = visualize_detections(frame, \n",
        "                                                 detections[0].boxes.cpu().xyxy, \n",
        "                                                 detections[0].boxes.cpu().conf, \n",
        "                                                 detections[0].boxes.cpu().cls)\n",
        "  output_video_boxes.write(frame_with_detections)\n",
        "\n",
        "  # Run frame and detections through SAM to get masks\n",
        "  transformed_boxes = mask_predictor.transform.apply_boxes_torch(detections[0].boxes.xyxy, list(get_video_dimensions(cap)))\n",
        "  if len(transformed_boxes) == 0:\n",
        "    print(\"No boxes found on frame\", frame_num)\n",
        "    output_video_masks.write(frame)\n",
        "    frame_num += 1\n",
        "    continue\n",
        "  mask_predictor.set_image(frame)\n",
        "  masks, scores, logits = mask_predictor.predict_torch(\n",
        "    boxes = transformed_boxes,\n",
        "    multimask_output=False,\n",
        "    point_coords=None,\n",
        "    point_labels=None\n",
        "  )\n",
        "  masks = np.array(masks.cpu())\n",
        "  if masks is None or len(masks) == 0:\n",
        "    print(\"No masks found on frame\", frame_num)\n",
        "    output_video_masks.write(frame)\n",
        "    frame_num += 1\n",
        "    continue\n",
        "  merged_colored_mask = merge_masks_colored(masks, detections[0].boxes.cls)\n",
        "  \n",
        "  # Write masks to output video\n",
        "  image_combined = cv2.addWeighted(frame, 0.7, merged_colored_mask, 0.7, 0)\n",
        "  output_video_masks.write(image_combined)\n",
        "\n",
        "  # Create video mask annotation for upload to Labelbox\n",
        "  instance_uri = get_instance_uri(client, global_key, merged_colored_mask)\n",
        "  mask_frame = create_mask_frame(frame_num, instance_uri)\n",
        "  mask_frames.append(mask_frame)\n",
        "  \n",
        "  frame_num += 1\n",
        "\n",
        "  # For the purposes of this demo, only look at the first 90 frames\n",
        "  if frame_num > 90:\n",
        "    break\n",
        "\n",
        "cap.release()\n",
        "output_video_boxes.release()\n",
        "output_video_masks.release()\n",
        "cv2.destroyAllWindows()"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Create annotations for LB upload\n",
        "mask_instances = create_mask_instances(unique_class_ids)\n",
        "annotations = []\n",
        "for instance in mask_instances:\n",
        "  annotations.append(create_video_mask_annotation(mask_frames, instance))\n",
        "\n",
        "labels = []\n",
        "labels.append(\n",
        "    lb_types.Label(data=lb_types.VideoData(global_key=global_key),\n",
        "                   annotations=annotations))"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "metadata": {},
      "source": [
        "# Upload the predictions to your specified project and data rows as pre-labels\n",
        "# Note: This may take a few minutes, depending on size of video and number of masks\n",
        "\n",
        "upload_job = lb.MALPredictionImport.create_from_objects(\n",
        "    client=client,\n",
        "    project_id=project.uid,\n",
        "    name=\"mal_import_job\" + str(uuid.uuid4()),\n",
        "    predictions=labels\n",
        ")\n",
        "upload_job.wait_until_done()\n",
        "print(f\"Errors: {upload_job.errors}\", )\n",
        "print(f\"Status of uploads: {upload_job.statuses}\")"
      ],
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    }
  ]
}